package spider.com.fenwe.test;

import java.io.File;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Random;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang3.StringUtils;

import com.sun.media.sound.UlawCodec;
import com.sun.org.apache.xml.internal.resolver.helpers.PublicId;
import com.sun.org.apache.xpath.internal.operations.And;

import spider.com.fenwe.util.HttpUtil;
import spider.com.fenwe.util.StreamReader;
import sun.print.resources.serviceui;

public class NetSpider {
    static String elemetRegex = "<a\\s*.*?</a>";
    static Pattern elementPattern = Pattern.compile(elemetRegex);
    static String urlRegex = "href\\s*=.*?(\\w.*?)[\\s+|>|\"]";
    static Pattern urlPattern = Pattern.compile(urlRegex);

    static String imgElementRegex = "<img.*?src\\s*?=.*?>";
    static Pattern imgElementPattern = Pattern.compile(imgElementRegex);
    static String imgSrcRegex = "(src|data-imgurl)=\".*?\"";
    static Pattern imgSrcPattern = Pattern.compile(imgSrcRegex);

    // 对于非标准img元素的图片链接提取
    static String nonStrdImgSrcRegex = "(http|https)://.*?\\.(jpg|png|bmp|jpeg)";
    static Pattern nonStrdImgSrcPattern = Pattern.compile(nonStrdImgSrcRegex);

    static String filePathSuffix = "F:\\NetSpider\\";
    static String dirName = "qzone";
    static String origUrl = "";

    public static void main(String[] args) {
        NetSpider netSpider = new NetSpider();
        // String filePathSuffix =
        // "C:\\Users\\Administrator\\Desktop\\NetSpider\\";
         String origUrl = "https://user.qzone.qq.com/1120874358";
//        origUrl = args[0];
//        dirName = origUrl.substring(7,10);
        File imgDir = new File(filePathSuffix + dirName + "\\image");
        if (!imgDir.exists()) {
            imgDir.mkdirs();
        }
        netSpider.grab(origUrl, 01);

    }

    // public static void mkDir(String)
    public String getHostPath(String url) {
        int index = url.indexOf("/", 8);
        return index == -1 ? url : url.substring(0, index);
    }

    public String completeUrl(String hostUrl, String url) {
        String temp = url.substring(8, url.indexOf("/", 8));
        if (temp.indexOf(".") == -1) {
            int index = url.indexOf("://");
            return getHostPath(hostUrl) + "/" + url.substring(index + 3);
        }
        return url;
    }

    public void grab(String origUrl, int flag) {
        Set<String> origSet = new HashSet<>();
        origSet.add(origUrl);
        Set<String> glbUrlList = new HashSet<>();
        Set<String> imgNameSet = new HashSet<>();
        grabRecursion(origSet, glbUrlList, imgNameSet, 1);
    }

    public void grabRecursion(Set<String> urlSrcList, Set<String> glbUrlList, Set<String> imgNameSet, long step) {

        if (urlSrcList != null && urlSrcList.size() > 0 && step < 100000000000l) {
            Set<String> urlList = new HashSet<>();
            for (String urlSrc : urlSrcList) {
                String result = null;
                try {
                    System.out.println(urlSrc);
                    Thread.sleep(100);
                     result = HttpUtil.get(urlSrc);
//                    result = HttpUtil.getAjaxCotnent(urlSrc);
                    if (StringUtils.isNotEmpty(result)) {
                        System.out.println(result);
                        // StreamReader.fileOutputStream(convertUrl2FilePath(urlSrc),
                        // result);
                        String resultCopy = result;
                        Matcher imgElementMatcher = imgElementPattern.matcher(result);
                        while (imgElementMatcher.find()) {
                            String element = imgElementMatcher.group(0);
                            Matcher imgUrlMatcher = imgSrcPattern.matcher(element);
                            if (imgUrlMatcher.find()) {
                                String url = imgUrlMatcher.group(0);
                                System.out.println("imgUrl-----" + url);
                                resultCopy.replaceAll(url, "");
                                dealImgUrl(url, urlSrc, imgNameSet);
                            }
                        }

                        // 对于非标准img元素的图片链接提取
                        Matcher nonStrdImgSrcMatcher = nonStrdImgSrcPattern.matcher(resultCopy);
                        while (nonStrdImgSrcMatcher.find()) {
                            String element = nonStrdImgSrcMatcher.group(0);
                            dealImgUrl(element, urlSrc, imgNameSet);
                        }
                        Matcher elementMatcher = elementPattern.matcher(result);
                        while (elementMatcher.find()) {
                            String element = elementMatcher.group(0);
                            Matcher urlMatcher = urlPattern.matcher(element);
                            if (urlMatcher.find()) {
                                String url = urlMatcher.group(1);
                                if (url.endsWith("/")) {
                                    System.out.println("-----" + url);
                                    url = url.substring(0, url.length() - 1);
                                }
                                if (!url.startsWith("http")) {
                                    url = getHostPath(urlSrc) + "/" + url;
                                    System.out.println("host complete-----" + url);
                                }
                                urlList.add(url);
                            }
                            // System.out.println("Found value: " +
                            // elementMatcher.group(0));
                        }

                    }
                } catch (Exception e) {
                    System.out.println(urlSrc);
                    // e.printStackTrace();
                }
            }
            Set<String> retain = new HashSet<String>(urlList);
            retain.retainAll(glbUrlList);
            urlList.removeAll(retain);
            glbUrlList.addAll(urlList);
            grabRecursion(urlList, glbUrlList, imgNameSet, step++);
        }
    }

    public Set<String> dealImgUrl(String url, String urlSrc, Set<String> imgNameSet) throws InterruptedException {
        int index = url.indexOf("http");
        if (index == 0) {
        } else if (index > -1) {
            url = url.substring(url.indexOf("http"), url.length() - 1);
            // 对是通过动态补全主网址的加上主网址前缀
            url = completeUrl(urlSrc, url);
            System.out.println("completed-----" + url);

        } else if (url.indexOf("src=\"/") != -1) {
            url = getHostPath(urlSrc) + url.substring(5);
            System.out.println("start with src=\"/----" + url);
        } else {
            return imgNameSet;
        }
        String imgName = url.substring(url.lastIndexOf("/") + 1);
        imgName = imgName.length() > 24 ? imgName.substring(imgName.length() - 25) : imgName;
        imgName = imgName.indexOf(".") == -1 ? imgName + ".jpg" : imgName;
        if (!imgNameSet.contains(imgName)) {
            Thread.sleep(100);
            HttpUtil.writeImageToDisk(url, filePathSuffix + dirName + "\\image\\" + imgName);
            imgNameSet.add(imgName);
            System.out.println("imgName-----" + imgName);
        }
        return imgNameSet;
    }

    public String convertUrl2FilePath(String urlSrc) {
        String filePath = null;
        int httpDvd = urlSrc.indexOf("//");
        int urlDvd = urlSrc.indexOf("/", httpDvd + 2);
        if (urlSrc.indexOf("//") != -1) {
            String cvtedUrl = urlSrc.substring(urlSrc.indexOf("//") + 2).replace("/", "_").replace("?", "@")
                    .replace("\"", "!");
            if (cvtedUrl.length() > 255) {
                cvtedUrl = cvtedUrl.substring(0, 254);
            }
            // if (urlDvd != -1) {
            // System.out.println(urlSrc);
            // System.out.println(urlDvd);
            filePath = filePathSuffix + dirName + "\\" + cvtedUrl + ".html";
            // } else {
            // filePath = filePathSuffix + urlSrc.substring(urlSrc.indexOf("//")
            // + 2) + ".html";
            // }
        } else {
            String cvtedUrl = urlSrc.replace("/", "_").replace("?", "@");
            if (cvtedUrl.length() > 255) {
                cvtedUrl = cvtedUrl.substring(0, 254);
            }
            filePath = filePathSuffix + cvtedUrl + ".html";
            // if (urlDvd != -1) {
            // filePath = filePathSuffix + urlSrc.substring(0, urlDvd - 1) +
            // ".html";
            // } else {
            // filePath = filePathSuffix + urlSrc + ".html";
            // }
        }
        return filePath;
    }
}
